package com.springboot.service.jubt;

import cn.hutool.json.JSONArray;
import cn.hutool.json.JSONObject;
import cn.hutool.json.JSONUtil;
import com.alibaba.fastjson.JSON;
import us.codecraft.webmagic.ResultItems;
import us.codecraft.webmagic.Task;
import us.codecraft.webmagic.pipeline.Pipeline;

import java.io.FileWriter;
import java.io.IOException;
import java.nio.file.Files;
import java.nio.file.Paths;
import java.util.ArrayList;
import java.util.HashSet;
import java.util.List;
import java.util.Set;
import java.util.concurrent.CopyOnWriteArrayList;

/**
 * @Classname SingleJsonFilePipeline
 * @Description TODO
 * @Version 1.0.0
 * @Date 2025/2/10 10:35
 * @Author by zyj
 */
public class SingleJsonFilePipeline implements Pipeline {

    private String filePath;
    private List<Object> dataList = new CopyOnWriteArrayList<>(); // 线程安全集合

    public SingleJsonFilePipeline(String filePath) {
        this.filePath = filePath;
    }

    @Override
    public void process(ResultItems resultItems, Task task) {
        Object data = resultItems.get("results");
        if (data != null) {
            dataList.add(data);
//            System.out.println("已收集数据: " + data);
        }
    }

    public void close() {
        try {
            // 确保目录存在
            Files.createDirectories(Paths.get(filePath).getParent());
            // 写入文件
            try (FileWriter writer = new FileWriter(filePath)) {
                Set<JSONObject> setChildren = new HashSet<>();
                Set<JSONObject> setWeb = new HashSet<>();

                JSONArray json = JSONUtil.parseArray(dataList);
                for(int i=0;i<json.size();i++){
                    JSONArray jsonArray1 = json.getJSONArray(i);
                    for(int j=0;j<jsonArray1.size();j++){
                        JSONObject jsonObject = jsonArray1.getJSONObject(j);
                        if(jsonObject.isEmpty()){
                            continue;
                        }
                        boolean web = jsonObject.getJSONArray("web")==null||jsonObject.getJSONArray("web").isEmpty()?false:true;
                        boolean children = jsonObject.getJSONArray("children")==null||jsonObject.getJSONArray("children").isEmpty()?false:true;
                        if( !web && !children ){
                            continue;
                        }
                        if(children){
                            setChildren.add(jsonObject);
                        }

                        if(web){
                            setWeb.add(jsonObject);
                        }
                    }
                }

                JSONArray resultAll = new JSONArray();
                resultAll.addAll(setWeb);
                resultAll.addAll(setChildren);
                writer.write(resultAll.toString());
                System.out.println("数据: " + resultAll.toString());
                System.out.println("数据已保存到: " + filePath);
                System.out.println("总数据量: " + resultAll.size());
            }
        } catch (IOException e) {
            System.err.println("保存文件失败: " + e.getMessage());
            e.printStackTrace();
        }
    }
}
